package edu.hawaii.webspider;

import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.logging.Logger;
import com.meterware.httpunit.WebLink;
import com.meterware.httpunit.WebConversation;
import com.meterware.httpunit.WebRequest;
import com.meterware.httpunit.WebResponse;

/**
 * A demonstration class illustrating how to retrieve and process web pages
 * using HttpUnit. This code is highly inspired by Edward Kim. Also small code
 * segments were written by Randy Cox and Jared Sunouchi
 * 
 * @author Philip Johnson and Hong Nguyen
 */
public class WebSpiderExample {

  /** Contains the starting url for the crawl. */
  private final String startUrl;
  private final List<String> usedURL = new ArrayList<String>();
  private static final List<String> linkList = new ArrayList<String>();
  private static int NumberOfPagesToVisit = 0;
  private static final Map<String, Integer> popularPages = new HashMap<String, Integer>();
  private static boolean log = false;
  private static Logger logOut = Logger.getLogger("WebSpiderExample.log");
  private int numOfTraverses = 0;
  private int position = 0;
  private static int mostPopularLinkTotal = 0;

  /**
   * A WebSpider traverses the page links and returns info.
   * 
   * @param startUrl
   *                The home url for the crawl.
   */
  public WebSpiderExample(String startUrl) {
    this.startUrl = startUrl;
  }

  /**
   * Returns the number of links found at the startUrl.
   * 
   * @return The number of links.
   * @throws Exception
   *                 if problems occur retrieving the startUrl.
   */
  public int getNumLinks() throws Exception {
    // keeps Javascript error from popping up.
    com.meterware.httpunit.HttpUnitOptions
        .setExceptionsThrownOnScriptError(false);
    com.meterware.httpunit.HttpUnitOptions
        .setExceptionsThrownOnErrorStatus(false);
    com.meterware.httpunit.HttpUnitOptions.setScriptingEnabled(false);

    // the array that will hold all the links.
    WebLink[] pagelinks = new WebLink[1000];
    ArrayList<String> linkArray = new ArrayList<String>();
    int totalLinks = 0;

    // create the conversation object which will maintain state for us
    WebConversation wc = new WebConversation();
    try {
      WebResponse resp = wc.getResponse(this.startUrl);
      usedURL.add(this.startUrl);
      pagelinks = resp.getLinks();
      totalLinks = resp.getLinks().length;
      if (log) {
        logOut.info("/n");
        logOut.info("Retrieving: " + this.startUrl);
        logOut.info("Found " + totalLinks + " links.");
      }
    }

    catch (Exception e) {
      if (log) {
        logOut.info("Error link is either broken or unaccessable.");
      }
    }

    for (WebLink transLink : pagelinks) {
      // Taken from Randy Cox.
      WebRequest wbTemp = transLink.getRequest();
      URL urlTemp;
      urlTemp = wbTemp.getURL();
      String urlString = urlTemp.toString();
      linkArray.add(urlString);
      linkList.add(urlString);
    }

    numOfTraverses = 1;
    totalLinks = traverseLinks(linkArray, totalLinks);

    return totalLinks;
  }

  /**
   * Main method of the Webspider program.
   * 
   * 
   * @param args
   *                Ignored.
   * @throws Exception
   *                 If problems occur.
   */
  public static void main(String[] args) throws Exception {

    if ((args.length != 0) && args[0].equals("-totallinks")) {

      if (args[args.length - 1].equals("-logging")) {
        log = true;
        logOut.info("Finding the total number of links...");
      }

      NumberOfPagesToVisit = Integer.parseInt(args[2]);

      WebSpiderExample example = new WebSpiderExample(args[1]);
      System.out.println("The total number of links are : "
          + example.getNumLinks());

    }

    else if ((args.length != 0) && args[0].equals("-mostpopular")) {

      try {
        if (args[3].equals("-logging")) {
          log = true;
          logOut.info("Calculating most popular page...");
        }
      }
      catch (ArrayIndexOutOfBoundsException e) {
        if (log) {
          logOut.info("Error there was no logging");
        }
      }
      NumberOfPagesToVisit = Integer.parseInt(args[2]);

      WebSpiderExample example = new WebSpiderExample(args[1]);
      example.getNumLinks();
      mostPopular(linkList);
      System.out.println("The most popular page discovered is: "
          + WebSpiderExample.mostPopular(linkList) + " with "
          + mostPopularLinkTotal + " pages that point to it.");
    }
  }

  /**
   * Retrieves the the start URL home page and counts the number of links on it.
   * Then traverses the to another URl that is connected to the current URL.
   * 
   * @param linkArray
   *                an array of links
   * @param totalLinks
   *                The total amount of links.
   * @throws Exception
   *                 If problems occur.
   * @return totalLinks The total amount of links.
   */
  public int traverseLinks(List<String> linkArray, int totalLinks)
      throws Exception {

    // keeps Javascript error from popping up.
    com.meterware.httpunit.HttpUnitOptions
        .setExceptionsThrownOnScriptError(false);
    com.meterware.httpunit.HttpUnitOptions
        .setExceptionsThrownOnErrorStatus(false);
    com.meterware.httpunit.HttpUnitOptions.setScriptingEnabled(false);

    WebLink[] pagelinks = new WebLink[1000];
    int count = 0;
    int totalLinksVisited = totalLinks;

    if (position < linkArray.size() && numOfTraverses < NumberOfPagesToVisit) {
      String currentLink = linkArray.get(position);

      while (usedURL.contains(currentLink)) {
        currentLink = linkArray.get(position++);
      }

      WebConversation wc = new WebConversation();

      try {
        WebResponse resp = wc.getResponse(currentLink);
        pagelinks = resp.getLinks();
        totalLinksVisited += resp.getLinks().length;
        count = resp.getLinks().length;
        if (log) {
          logOut.info("/n");
          logOut.info("Current Page: " + this.startUrl);
          logOut.info("Found total of: " + count + " links.");
        }
      }
      catch (Exception e) {
        if (log) {
          logOut.info("Error the link is either broken or unaccessable.");
        }
      }

      usedURL.add(currentLink);

      if (count > 1) {
        for (WebLink transLink : pagelinks) {
          // Taken from Randy Cox.
          WebRequest wbTemp = transLink.getRequest();
          URL urlTemp;
          urlTemp = wbTemp.getURL();
          String urlString = urlTemp.toString();
          linkArray.add(urlString);
          linkList.add(urlString);
        }

      }

      // increments the number of pages visited.
      numOfTraverses++;

      totalLinksVisited = traverseLinks(linkArray, totalLinksVisited);

      return totalLinksVisited;
    }

    else {
      return totalLinksVisited;
    }

  }

  /**
   * This method will take a link list containing URL which computes and returns
   * the most popular URL.
   * 
   * @param linkList
   *                All the links found
   * @return popular The most popular website.
   */
  public static String mostPopular(List<String> linkList) {
    // This code is written by Jared Sunouchi
    String popular = "";

    for (String transferLink : linkList) {
      int count = 1;
      if (popularPages.containsKey(transferLink)) {
        count += popularPages.get(transferLink);
      }
      popularPages.put(transferLink, count);
    }

    Set<Map.Entry<String, Integer>> mapSet = popularPages.entrySet();
    for (Map.Entry<String, Integer> page : mapSet) {
      if (page.getValue().intValue() > mostPopularLinkTotal) {
        mostPopularLinkTotal = page.getValue().intValue();
        popular = page.getKey();
      }
    }

    return popular;
  }

}